/*
 * Copyright (c) 2018-2024 北京华恒盛世科技有限公司
 * QUICKPOOL is licensed under Mulan PSL v2.
 * You can use this software according to the terms and conditions of the Mulan PSL v2.
 * You may obtain a copy of Mulan PSL v2 at:
 *     http://license.coscl.org.cn/MulanPSL2
 * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
 * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
 * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
 * See the Mulan PSL v2 for more details.
*/

#include "jm.h"

static int execute_terminate_command(struct jobEntry *, int, int, int, bool_t);
static int execute_resume_command(struct jobEntry *, int , int, bool_t);
static int execute_signal_command(struct jobEntry *, int, char *, int);

static int kill_jobpg(struct jobEntry *, int);

/**
 * @brief       Send signal to job and execute relative action script.
 *
 * @param[in]   job          #1: job entry
 * @param[in]   signal       #2: signal to be sent to job
 * @param[in]   actflags     #3: action script flag
 * @param[in]   reportjs     #4: if report job status to JS
 *
 * @retval      0            #1: succeed
 * @retval      -1           #2: failed
 *
 * @note        It is time to send signal to job processes.
 */
int
signal_job_begin(struct jobEntry *job, int signal, int actflags, bool_t reportjs)
{
    char strJobId[32];
    char strSignal[30];

    qpsignal_2string(signal, strSignal);
    log_message(DEBUG, DEBUGSIGNAL, "%s: signal %s to job %s with actflags=%d, execute_flag %d act_pid %d reasons=%d subresons=%d signal=%d",
                __func__, strSignal, jobid_2string(job->job_base.jobid, strJobId), actflags,
                job->execute_flag, job->job_base.act_pid, job->job_base.reasons,
                job->job_base.subreasons, signal);

    int defSigValue;
    defSigValue = signal;

    if (!(job->job_base.status & JOB_STATUS_ZOMBIE) && IS_JOB_ENDED(job->job_base.status)) {
        return 0;
    }

    int cc;
    char *actCmd;
    if (!JOB_STARTED(job)) {
        switch(signal) {
        case SIGKILL:
            job->actcmd_state = ACT_NO;
            return (send_signal_2job(job, SIGKILL, TRUE));
        case QUICKPOOL_SIGKILL_USER:
        case QUICKPOOL_SIGKILL_REQUEUE:
        case QUICKPOOL_SIGKILL_FORCE:
            return (execute_terminate_command(job, signal, SUSPEND_REASON_USER_STOP, 0, reportjs));
        case QUICKPOOL_SIGKILL_OTHER:
            return (execute_terminate_command(job, signal,
                                              job->susp_reason, job->susp_subreason, reportjs));
        case QUICKPOOL_SIGKILL_RUNLIMIT:
            return (execute_terminate_command(job, signal,
                                              SUSPEND_REASON_RESLIMIT, SUBREASON_RUNTIME, reportjs));
        case QUICKPOOL_SIGKILL_DEADLINE:
            return (execute_terminate_command(job, signal,
                                              SUSPEND_REASON_RESLIMIT, SUBREASON_DEADLINE, reportjs));
        case QUICKPOOL_SIGKILL_PROCESSLIMIT:
            return (execute_terminate_command(job, signal,
                                              SUSPEND_REASON_RESLIMIT, SUBREASON_PROCESS, reportjs));
        case QUICKPOOL_SIGKILL_CPULIMIT:
            return (execute_terminate_command(job, signal,
                                              SUSPEND_REASON_RESLIMIT, SUBREASON_CPUTIME, reportjs));
        case QUICKPOOL_SIGKILL_MEMLIMIT:
            return (execute_terminate_command(job, signal,
                                              SUSPEND_REASON_RESLIMIT, SUBREASON_MEMORY, reportjs));
        default:
            return -1;
	}
    }

    if ((0 < job->job_base.act_pid) && is_terminate_signal(job->job_base.act_signal)) {
        // job action script is running
        return -1;
    }

    if (job->job_base.act_pid || (job->job_base.status & JOB_STATUS_MIG)) {

        switch(signal) {
        case SIGKILL:

            job->actcmd_state = ACT_NO;
            if ((cc = send_signal_2job(job, SIGKILL, TRUE)) >= 0) {
                job->job_base.status &= ~JOB_STATUS_MIG;
            }
            return cc;
        case QUICKPOOL_SIGKILL_USER:
        case QUICKPOOL_SIGKILL_REQUEUE:
        case QUICKPOOL_SIGKILL_FORCE:
            return (execute_terminate_command(job, signal, SUSPEND_REASON_USER_STOP, 0, reportjs));
        case QUICKPOOL_SIGKILL_OTHER:
            return (execute_terminate_command(job, signal,
                                              job->susp_reason, job->susp_reason, reportjs));
        case QUICKPOOL_SIGKILL_RUNLIMIT:
            return (execute_terminate_command(job, signal,
                                              SUSPEND_REASON_RESLIMIT, SUBREASON_RUNTIME, reportjs));
        case QUICKPOOL_SIGKILL_DEADLINE:
            return (execute_terminate_command(job, signal,
                                              SUSPEND_REASON_RESLIMIT, SUBREASON_DEADLINE, reportjs));
        case QUICKPOOL_SIGKILL_PROCESSLIMIT:
            return (execute_terminate_command(job, signal,
                                              SUSPEND_REASON_RESLIMIT, SUBREASON_PROCESS, reportjs));
        case QUICKPOOL_SIGKILL_CPULIMIT:
            return (execute_terminate_command(job, signal,
                                              SUSPEND_REASON_RESLIMIT, SUBREASON_CPUTIME, reportjs));
        case QUICKPOOL_SIGKILL_MEMLIMIT:
            return (execute_terminate_command(job, signal,
                                              SUSPEND_REASON_RESLIMIT, SUBREASON_MEMORY, reportjs));
        default:
            return -1;
        }
    }

    if ((job->job_base.status & JOB_STATUS_RUN) && job->post_started) {
        // job has been done, post script is running.
        return -1;
    }

    if (job->job_base.status & JOB_STATUS_RUN) {
        if (0 <= signal) {

            job->actcmd_state = ACT_NO;
            switch(signal) {
            case SIGSTOP:
            case SIGTSTP:
            case SIGTTIN:
            case SIGTTOU:
                if (send_signal_2job(job, signal, TRUE) < 0) {
                    return -1;
                }

                SET_STATE(job->job_base.status, JOB_STATUS_USUSP);
                job->job_base.reasons |= SUSPEND_REASON_USER_STOP;
                job->comm_failcnt ++;

                return 0;
            default:
                return (send_signal_2job(job, signal, TRUE));
            }
        }

        switch (signal) {
        case QUICKPOOL_SIGCHKPNT:
            job->actcmd_state = ACT_NO;
            if ((cc = execute_signal_command(job, signal, NULL, actflags)) < 0) {
                return cc;            
            }

            job->actcmd_state = ACT_START;
            job->job_base.status |= JOB_STATUS_SIGNAL;

            if (actflags & QP_CHKPNT_MIG)
                job->job_base.status |= JOB_STATUS_MIG;

            if (reportjs) {
                if (status_job(BATCH_STATUS_JOB, job, job->job_base.status, 0) < 0) {
                    job->comm_failcnt ++;
                    return cc;
                }

                if (0 < job->comm_failcnt) {
                    job->comm_failcnt = 0;
                }

                job->actcmd_state = ACT_NO;
            }

            return cc;            
        case QUICKPOOL_SIGSTOP_USER:
            return (execute_suspend_command(job, signal, JOB_STATUS_USUSP,
                                            SUSPEND_REASON_USER_STOP, 0, reportjs));
        case QUICKPOOL_SIGSTOP_LOAD:
            return (execute_suspend_command(job, signal, JOB_STATUS_SSUSP,
                                            job->susp_reason, job->susp_subreason, reportjs));
        case QUICKPOOL_SIGSTOP_PREEMPT:
            return (execute_suspend_command(job, signal, JOB_STATUS_SSUSP,
                                            SUSPEND_REASON_PREEMPTED, 0, reportjs));
        case QUICKPOOL_SIGSTOP_WINDOW:
            return (execute_suspend_command(job, signal, JOB_STATUS_SSUSP,
                                            SUSPEND_REASON_QUEUE_WINDOW, 0, reportjs));
        case QUICKPOOL_SIGSTOP_OTHER:
            return (execute_suspend_command(job, signal, JOB_STATUS_SSUSP,
                                            job->susp_reason, job->susp_subreason, reportjs));
        case QUICKPOOL_SIGCONT_USER:
        case QUICKPOOL_SIGCONT_LOAD:
        case QUICKPOOL_SIGCONT_WINDOW:
        case QUICKPOOL_SIGCONT_OTHER:

            job->actcmd_state = ACT_NO;
            actCmd = job->job_base.resume_command;
            defSigValue = get_signal_value(signal, actCmd);

            if (signal != defSigValue) {

                return (send_signal_2job(job, defSigValue, ((defSigValue == SIGKILL) ? TRUE : FALSE)));
            }

            if ((cc = execute_signal_command(job, signal, actCmd, actflags)) < 0) {
                return cc;
            }

            job->actcmd_state = ACT_START;
            job->job_base.status |= JOB_STATUS_SIGNAL;

            if (reportjs) {

                if (status_job(BATCH_STATUS_JOB, job, job->job_base.status, 0) < 0) {
                    job->comm_failcnt ++;
                    return cc;
                }

                if (0 < job->comm_failcnt) {
                    job->comm_failcnt = 0;
                }

                job->actcmd_state = ACT_NO;
            }

            return cc;
        case QUICKPOOL_SIGKILL_USER:
        case QUICKPOOL_SIGKILL_REQUEUE:
        case QUICKPOOL_SIGKILL_FORCE:
            return (execute_terminate_command(job, signal, SUSPEND_REASON_USER_STOP, 0, reportjs));
        case QUICKPOOL_SIGKILL_OTHER:
            return (execute_terminate_command(job, signal,
                                              job->susp_reason, job->susp_subreason, reportjs));
        case QUICKPOOL_SIGKILL_RUNLIMIT:
            return (execute_terminate_command(job, signal,
                                              SUSPEND_REASON_RESLIMIT, SUBREASON_RUNTIME, reportjs));
        case QUICKPOOL_SIGKILL_DEADLINE:
            return (execute_terminate_command(job, signal,
                                              SUSPEND_REASON_RESLIMIT, SUBREASON_DEADLINE, reportjs));
        case QUICKPOOL_SIGKILL_PROCESSLIMIT:
            return (execute_terminate_command(job, signal,
                                              SUSPEND_REASON_RESLIMIT, SUBREASON_PROCESS, reportjs));
        case QUICKPOOL_SIGKILL_CPULIMIT:
            return (execute_terminate_command(job, signal,
                                              SUSPEND_REASON_RESLIMIT, SUBREASON_CPUTIME, reportjs));
        case QUICKPOOL_SIGKILL_MEMLIMIT:
            return (execute_terminate_command(job, signal,
                                              SUSPEND_REASON_RESLIMIT, SUBREASON_MEMORY, reportjs));
        }
    }

    if (IS_JOB_SUSPEND(job->job_base.status)) {
        if (0 <= signal) {

            job->actcmd_state = ACT_NO;
            send_signal_2job(job, SIGCONT, FALSE);

            return (send_signal_2job(job, signal, TRUE));
	}

        switch (signal) {
        case SIGSTOP:
        case SIGTSTP:
        case SIGTTIN:
        case SIGTTOU:

            job->actcmd_state = ACT_NO;
            SET_STATE(job->job_base.status, JOB_STATUS_USUSP);
            job->job_base.reasons |= SUSPEND_REASON_USER_STOP;
            job->comm_failcnt ++;
            return 0;
        case QUICKPOOL_SIGSTOP_USER:
            SET_STATE(job->job_base.status, JOB_STATUS_USUSP);
            job->job_base.reasons |= SUSPEND_REASON_USER_STOP;
            job->comm_failcnt ++;
            return 0;
        case QUICKPOOL_SIGSTOP_LOAD:
            job->job_base.reasons |= job->susp_reason;
            job->job_base.subreasons = job->susp_subreason;
            job->comm_failcnt ++;
            return 0;
        case QUICKPOOL_SIGSTOP_PREEMPT:
            job->job_base.reasons |= SUSPEND_REASON_PREEMPTED;
            job->comm_failcnt ++;
            return 0;
        case QUICKPOOL_SIGSTOP_WINDOW:
            job->job_base.reasons |= SUSPEND_REASON_QUEUE_WINDOW;
            job->comm_failcnt ++;
            return 0;
        case QUICKPOOL_SIGSTOP_OTHER:
            job->job_base.reasons |= job->susp_reason;
            job->job_base.subreasons = job->susp_subreason;
            job->comm_failcnt ++;
            return 0;
        case QUICKPOOL_SIGCONT_USER:
            return (execute_resume_command(job, signal, SUSPEND_REASON_USER_STOP, reportjs));
        case QUICKPOOL_SIGCONT_LOAD:
            return (execute_resume_command(job, signal, (LOAD_REASONS), reportjs));
        case QUICKPOOL_SIGCONT_WINDOW:
            return (execute_resume_command(job, signal, SUSPEND_REASON_QUEUE_WINDOW, reportjs));
        case QUICKPOOL_SIGCONT_OTHER:
            return (execute_resume_command(job, signal, SUSPEND_REASON_RESOURCE, reportjs));
        case QUICKPOOL_SIGKILL_USER:
        case QUICKPOOL_SIGKILL_REQUEUE:
        case QUICKPOOL_SIGKILL_FORCE:
            return (execute_terminate_command(job, signal, SUSPEND_REASON_USER_STOP, 0, reportjs));
        case QUICKPOOL_SIGKILL_OTHER:
            return (execute_terminate_command(job, signal,
                                              job->susp_reason, job->susp_subreason, reportjs));
        case QUICKPOOL_SIGKILL_RUNLIMIT:
            return (execute_terminate_command(job, signal,
                                              SUSPEND_REASON_RESLIMIT, SUBREASON_RUNTIME, reportjs));
        case QUICKPOOL_SIGKILL_DEADLINE:
            return (execute_terminate_command(job, signal,
                                              SUSPEND_REASON_RESLIMIT, SUBREASON_DEADLINE, reportjs));
        case QUICKPOOL_SIGKILL_PROCESSLIMIT:
            return (execute_terminate_command(job, signal,
                                              SUSPEND_REASON_RESLIMIT, SUBREASON_PROCESS, reportjs));
        case QUICKPOOL_SIGKILL_CPULIMIT:
            return (execute_terminate_command(job, signal,
                                              SUSPEND_REASON_RESLIMIT, SUBREASON_CPUTIME, reportjs));
        case QUICKPOOL_SIGKILL_MEMLIMIT:
            return (execute_terminate_command(job, signal,
                                              SUSPEND_REASON_RESLIMIT, SUBREASON_MEMORY, reportjs));
        }
    }

    return 0;
} // end function signal_job_begin

/**
 * @brief       Take action to suspend a job.
 *
 * @param[in]   job          #1: job entry
 * @param[in]   signal       #2: signal value to suspend a job
 * @param[in]   status       #3: JOB_STATUS_USUSP or JOB_STATUS_SSUSP
 * @param[in]   reasons      #4: reason to supsend the job
 * @param[in]   subreasons   #5: detail reason (resource index) to suspend job
 * @param[in]   reportjs     #6: if report job status to JS
 *
 * @retval      0            #1: succeed
 * @retval      -1           #2: failed
 *
 * @note        Send susp signal to job if input signal is not the default one,
 *              otherwise execute defined suspend script.
 */
int
execute_suspend_command(struct jobEntry *job, int signal, int status,
                        int reasons, int subreasons, bool_t reportjs)
{
    job->susp_reason = reasons;
    job->susp_subreason = subreasons;

    job->actcmd_state = ACT_NO;

    char *strSuspCmd;
    strSuspCmd = job->job_base.suspend_command;

    int defSigValue;
    defSigValue = get_signal_value(signal, strSuspCmd);

    int ret;
    if (signal != defSigValue) {

        ret = send_signal_2job(job, defSigValue, TRUE);
        if (0 <= ret) {

            SET_STATE(job->job_base.status, status);
            job->job_base.reasons |= reasons;
            job->job_base.subreasons = subreasons;
            job->job_base.suspend_time = time(NULL);
            job->comm_failcnt ++;
        }

        return ret;
    }

    ret = execute_signal_command(job, signal, strSuspCmd, 0);
    if (0 > ret) {
        return ret;
    }

    job->actcmd_state = ACT_START;
    job->job_base.status |= JOB_STATUS_SIGNAL;

    if (reportjs) {

        if (status_job(BATCH_STATUS_JOB, job, job->job_base.status, 0) < 0) {
            job->comm_failcnt ++;
            return ret;
        }

        if (0 < job->comm_failcnt) {
            job->comm_failcnt = 0;
        }

        job->actcmd_state = ACT_NO;
    }

    return ret;
} // end function execute_suspend_command

/**
 * @brief       Take action to resume a job.
 *
 * @param[in]   job          #1: job entry
 * @param[in]   signal       #2: signal to be sent to job
 * @param[in]   reportjs     #3: if report job status to JS
 *
 * @retval      0            #1: succeed
 * @retval      -1           #2: failed
 *
 * @note        Send cont signal to job if input signal is not the default one,
 *              otherwise execute defined suspend script.
 */
int
take_resume_action(struct jobEntry *job, int signal, bool_t reportjs)
{
    int defSigValue;
    char *actCmd;
    job->actcmd_state = ACT_NO;
    actCmd = job->job_base.resume_command;
    defSigValue = get_signal_value(signal, actCmd);

    int cc;
    if (signal != defSigValue) {

        if ((cc = send_signal_2job(job, defSigValue,
                                   ((defSigValue == SIGKILL) ? TRUE : FALSE))) >= 0) {

            SET_STATE(job->job_base.status, JOB_STATUS_RUN);
            job->job_base.reasons = 0;
            job->comm_failcnt ++;
        }

        return cc;
    }

    if ((cc = execute_signal_command(job, signal, actCmd, 0)) < 0) {
        return cc;
    }

    job->actcmd_state = ACT_START;
    job->job_base.status |= JOB_STATUS_SIGNAL;

    if (reportjs) {

        if (status_job(BATCH_STATUS_JOB, job, job->job_base.status, 0) < 0) {
            job->comm_failcnt ++;
            return cc;
        }

        if (0 < job->comm_failcnt) {
            job->comm_failcnt = 0;
        }

        job->actcmd_state = ACT_NO;
    }

    return cc;
} // end function take_resume_action

/**
 * @brief       Send signal to job process.
 *
 * @param[in]   job          #1: job entry
 * @param[in]   sig          #2: signal to be sent to job
 * @param[in]   fork2sig     #3: flag to indicate fork a child to send the signal
 *
 * @retval      0            #1: succeed
 * @retval      -1           #2: failed, job process is gone
 *
 * @note        Could define SIGNAL_SIGSTOP to overwrite the default SIGSTOP.
 */
int
send_signal_2job(struct jobEntry *job, int sig, bool_t fork2sig)
{
    char strJobId[32];
    log_message(DEBUG2, DEBUGSIGNAL, "%s: send signal %d to job %s",
                __func__, sig, jobid_2string(job->job_base.jobid, strJobId));

    if (sig == SIGSTOP) {
	int stopSig;

	if (jmParams[SIGNAL_SIGSTOP].string_value
            && (stopSig = gf_string_2signal(jmParams[SIGNAL_SIGSTOP].string_value)) > 0) {
	    sig = stopSig;
	} else {
	    if (job->job_base.number_node > 1)
		sig = SIGTSTP;
	}
    }

    if (sig != SIGKILL) {
        return (kill_jobpg(job, sig));
    }

    int preSig;
    preSig = SIGINT;

    if (kill_jobpg(job, SIGCONT) < 0) {
        return -1;
    }

    int pid = -1;
    if (fork2sig) {

        pid = fork();
        if (0 > pid) {
            log_message(ERR, NOCLASS, STR_JOB_FAIL_S_M, __func__,
                        jobid_2string(job->job_base.jobid, strJobId), "fork");
            return -1;
        }
    }

    if (!fork2sig || pid <= 0) {

        if (fork2sig && pid == 0) {

            if (setpgid(0, getpid()) < 0) {
                log_message(ERR, NOCLASS, STR_FUNC_D_FAIL_M, __func__, "setpgid", 0);
            }

            close_log();
            gf_child_chan_close(-1);
            finalize_tcl();

            char logfile[PATH_MAX+1];
            snprintf(logfile, PATH_MAX, "%s/jmchild.%s.log",
                     baseParams[BASE_LOGDIR].string_value, gptrMyHost->name);
            open_log(logfile, baseParams[BASE_LOG_MASK].string_value,
                     baseParams[BASE_JM_TIME_DEBUG].int_value);
        }

        if (SUBREASON_CPUTIME == job->susp_subreason) {
#ifdef SIGXCPU
            kill_jobpg(job, SIGXCPU);
	    gf_sleep(gintJobTerminateInterval*1000);
#endif
#ifdef SIGCPULIM
            kill_jobpg(job, SIGCPULIM);
	    gf_sleep(gintJobTerminateInterval*1000);
#endif
        }

        if (kill_jobpg(job, preSig) == 0) {

            gf_sleep(gintJobTerminateInterval*1000);
            if (kill(job->job_base.job_pid, 0) != 0) {
                if (fork2sig && pid == 0) {
                    exit(0);
                } else {
                    return 0;
                }
            }

            if (kill_jobpg(job, SIGTERM) == 0) {

                gf_sleep(gintJobTerminateInterval*1000);
                if (kill(job->job_base.job_pid, 0) != 0) {
                    if (fork2sig && pid == 0) {
                        exit(0);
                    } else {
                        return 0;
                    }
                }

                kill_jobpg(job, SIGKILL);
            }
        }

        if (fork2sig && 0 == pid) {
	    exit(0);
        }
    }

    return 0;
} // end function send_signal_2job

char *
get_chkpnt_dir(const char *name, char *chkpntdir)
{
    char *strEnvChkpntDir;
    strEnvChkpntDir = getenv("QP_ECHKPNTDIR");
    if (NULL == strEnvChkpntDir) {

        strEnvChkpntDir = baseParams[BASE_SERVERDIR].string_value;
        if (NULL == strEnvChkpntDir) {
            perror("Can't access echkpnt()");
            return NULL;
        }
    }

    strncpy(chkpntdir, strEnvChkpntDir, PATH_MAX);
    strncat(chkpntdir, name, PATH_MAX);

    if (access(chkpntdir, X_OK) < 0) {
        perror("Can't access checkpoint dir.");
        return NULL;
    }

    return(chkpntdir);
} // end function get_chkpnt_execution

char *
get_exitfile_suffix(int signal)
{
    switch (signal) {
    case QUICKPOOL_SIGSTOP_USER:
    case QUICKPOOL_SIGSTOP_LOAD:
    case QUICKPOOL_SIGSTOP_PREEMPT:
    case QUICKPOOL_SIGSTOP_WINDOW:
    case QUICKPOOL_SIGSTOP_OTHER:
        return ((char *)"suspend");
    case QUICKPOOL_SIGCONT_USER:
    case QUICKPOOL_SIGCONT_LOAD:
    case QUICKPOOL_SIGCONT_WINDOW:
    case QUICKPOOL_SIGCONT_OTHER:
        return ((char *)"resume");
    case QUICKPOOL_SIGKILL_USER:
    case QUICKPOOL_SIGKILL_OTHER:
    case QUICKPOOL_SIGKILL_RUNLIMIT:
    case QUICKPOOL_SIGKILL_DEADLINE:
    case QUICKPOOL_SIGKILL_PROCESSLIMIT:
    case QUICKPOOL_SIGKILL_FORCE:
    case QUICKPOOL_SIGKILL_REQUEUE:
    case QUICKPOOL_SIGKILL_CPULIMIT:
    case QUICKPOOL_SIGKILL_MEMLIMIT:
    case QUICKPOOL_SIGKILL_SWAPLIMIT:
        return ((char *)"terminate");
    default:
        return ((char *)"unknown");
    }
} // end function get_exitfile_suffix

/* ------------------------------------------------
 *
 *  below are static functions used in this file
 *
 * ------------------------------------------------ */

/**
 * @brief       Take action to terminate a job.
 *
 * @param[in]   job          #1: job entry
 * @param[in]   signal       #2: signal value to terminate a job
 * @param[in]   reasons      #3: reason to supsend the job
 * @param[in]   subreasons   #4: detail reason (resource index) to suspend job
 * @param[in]   reportjs     #5: if report job status to JS
 *
 * @retval      0            #1: succeed
 * @retval      -1           #2: failed
 *
 * @note        Send kill signal to job if input signal is not the default one,
 *              otherwise execute defined terminate script.
 */
static int
execute_terminate_command(struct jobEntry *job, int signal,
                          int reasons, int subreasons, bool_t reportjs)
{
    if (QUICKPOOL_SIGKILL_FORCE == signal) {
        job->job_base.submit->terminate_time = time(0);
        job->job_base.job_attribute |= JOB_FORCE_KILL;
    }

    if (QUICKPOOL_SIGKILL_REQUEUE == signal) {
	job->job_base.reasons |= EXIT_REQUEUE;
	signal = QUICKPOOL_SIGKILL_USER;
    }

    int defSigValue;
    char *actCmd;
    actCmd = job->job_base.terminate_command;
    defSigValue = get_signal_value(signal, actCmd);

    job->susp_reason = reasons;
    job->susp_subreason = subreasons;
    job->actcmd_state = ACT_NO;

    if (signal != defSigValue) {

	if (QUICKPOOL_SIGKILL_RUNLIMIT == signal) {
            send_signal_2job(job, SIGQUIT, FALSE);
	}

        return (send_signal_2job(job, defSigValue, TRUE));
    }

    int ret;
    ret = execute_signal_command(job, signal, actCmd, 0);
    if (0 > ret) {
        return ret;
    }

    job->actcmd_state = ACT_START;
    job->job_base.status |= JOB_STATUS_SIGNAL;

    if (reportjs) {

        if (status_job(BATCH_STATUS_JOB, job, job->job_base.status, 0) < 0) {
            job->comm_failcnt ++;
            return ret;
        }

        if (0 < job->comm_failcnt) {
            job->comm_failcnt = 0;
        }

        job->actcmd_state = ACT_NO;
    }

    return ret;
} // end function execute_terminate_command

/**
 * @brief       Take action to resume a job.
 *
 * @param[in]   job          #1: job entry
 * @param[in]   signal       #2: signal to be sent to job
 * @param[in]   reasons      #3: reason to supsend the job
 * @param[in]   reportjs     #4: if report job status to JS
 *
 * @retval      0            #1: succeed
 * @retval      -1           #2: failed
 *
 * @note        Send cont signal to job if input signal is not the default one,
 *              otherwise execute defined suspend script.
 */
static int
execute_resume_command(struct jobEntry *job, int signal, int reasons, bool_t reportjs)
{
    char strJobId[32];
    log_message(DEBUG, DEBUGSIGNAL, "%s: job %s base reasons %d signal %d suspend reasons %d",
                __func__, jobid_2string(job->job_base.jobid, strJobId),
                job->job_base.reasons, signal, reasons);

    job->actcmd_state = ACT_NO;

    if (job->job_base.reasons & SUSPEND_REASON_JSLOCK) {

        if (QUICKPOOL_SIGCONT_USER != signal) {
            return (take_resume_action(job, signal, reportjs));
        }

        if (job->susp_reason & ~(SUSPEND_REASON_JSLOCK | SUSPEND_REASON_USER_STOP
                                 | SUSPEND_REASON_USER_RESUME | SUSPEND_REASON_JMSTART)) {

            job->job_base.reasons = job->susp_reason & ~(SUSPEND_REASON_USER_STOP
                                                         | SUSPEND_REASON_USER_RESUME);
            SET_STATE(job->job_base.status, JOB_STATUS_SSUSP);
            return 0;
        }

        return (take_resume_action(job, signal, reportjs));
    }


    if (job->job_base.reasons & ~(reasons | SUSPEND_REASON_USER_RESUME | SUSPEND_REASON_JSLOCK)) {

        job->job_base.reasons &= ~reasons;
        if (QUICKPOOL_SIGCONT_USER == signal) {

            SET_STATE(job->job_base.status, JOB_STATUS_SSUSP);
            return 0;
        }

        return (take_resume_action(job, signal, reportjs));
    }

    return (take_resume_action(job, signal, reportjs));
} // end function execute_resume_command

/**
 * @brief       call kill to send signal to job.
 *
 * @param[in]   job          #1: job
 * @param[in]   sig          #2: signal to be sent
 *
 * @retval      0            #1: succeed
 * @retval      > 0          #2: kill return fail
 *
 * @note        if sig == 0, test if job pid has been gone.
 */
static int
kill_jobpg(struct jobEntry *job, int sig)
{
    int pgid0[1], *pgid;
    unsigned int i;
    
    char strJobId[32];
    log_message(DEBUG, DEBUGSIGNAL, "%s: Job <%s> pid %d pgid %d restart pid %d sig %d",
                __func__, jobid_2string(job->job_base.jobid, strJobId), job->job_base.job_pid,
                job->job_base.job_pgid, job->job_base.restart_pid, sig);

    if (0 == job->job_base.job_pgid) {
	if (kill(job->job_base.job_pid, sig) == 0) {
	    kill(-job->job_base.job_pid, sig);
	    return 0;
	}
	return (kill(-job->job_base.job_pid, sig));
    }

    if (0 == job->inst_usage[0].npgids) {
	pgid = pgid0;
	pgid[0] = job->job_base.job_pgid;
    } else {
	pgid = job->inst_usage[0].pgid;
    }

    struct jobResourceUsage *jru = NULL;
    RECORD_FUNCTIME(0, jru = cgroup_get_job_rusage(job->job_base.jobid), "cgroup_get_job_rusage");

    if (jru) {
        if (jru->npgids >0) {
            static int jmPgid = -1;

            if (jmPgid < 0) {
                jmPgid =  getpgrp();
            }

            for (i=0; i<jru->npgids; i++) {
                if (jmPgid == jru->pgid[i]) {

                    log_message(DEBUG, DEBUGSIGNAL, "%s: jm's pgid <%d> is the same as job <%s>",
                                __func__, jmPgid, jobid_2string(job->job_base.jobid, strJobId));
                    unsigned int j;

                    for (j=i; j < jru->npgids - 1; j++) {
                        jru->pgid[j] = jru->pgid[j+1];
                    }
                    jru->npgids--;
                }
            }

            if (jru->npgids == 0) {
                FREEUP(jru->pidinfo);
                FREEUP(jru->pgid);
                FREEUP(jru);
                return 0;
            }
        }

        if (jru->mem > job->max_usage[0].mem) {
            job->max_usage[0].mem = jru->mem;
        }

        if (jru->swap > job->max_usage[0].swap) {
            job->max_usage[0].swap = jru->swap;
        }

        if (jru->npids > job->max_usage[0].npids) {
            job->max_usage[0].npids = jru->npids;
        }

        if (jru->utime > job->max_usage[0].utime) {
            job->max_usage[0].utime = jru->utime;
        } else {
            jru->utime = job->max_usage[0].utime;
        }

        if (jru->stime > job->max_usage[0].stime) {
            job->max_usage[0].stime = jru->stime;
        } else {
            jru->stime = job->max_usage[0].stime;
        }

        time_t checkNow = time(NULL);
        if (checkNow - job->report_time > gintUpdateUsage * gintJMSleepTime) {

            log_message(DEBUG, DEBUGSIGNAL, "%s: will report job <%s> usage to JS soon.",
                        __func__, jobid_2string(job->job_base.jobid, strJobId));

            job->update_usage = TRUE;
        }

        copy_job_resourceusage(&(job->inst_usage[0]), jru);

        if (jru->npgids > 0) {
           FREEUP(jru->pgid);
        }

        if (jru->npids > 0) {
            FREEUP(jru->pidinfo);
        }
        FREEUP(jru);
    }

    if (kill(job->job_base.job_pid, sig) == 0) {

        log_message(DEBUG, DEBUGSIGNAL, "%s: Job %s kill(pid=%d) is good", 
                    __func__, jobid_2string(job->job_base.jobid, strJobId), job->job_base.job_pid);

        if (0 < job->inst_usage[0].npgids) {
            for (i = 0; i < job->inst_usage[0].npgids; i++) {
                kill(-job->inst_usage[0].pgid[i], sig);
            }
        } else {
            kill(-job->job_base.job_pgid, sig);
        }

        return 0;
    } else {
        log_message(DEBUG, DEBUGSIGNAL, "%s: Job %s kill(pid=%d) <%d> failed <%m>",
                    __func__, jobid_2string(job->job_base.jobid, strJobId),
                    job->job_base.job_pid, sig);
    }

    if (0 < job->inst_usage[0].npgids) {
        unsigned int ndead = 0;

        for (i = 0; i < job->inst_usage[0].npgids; i++) {

            if (kill(-job->inst_usage[0].pgid[i], sig) < 0) {

                ndead++;
                log_message(DEBUG, DEBUGSIGNAL, "%s: Job %s kill(%d) group are gone",
                            __func__, jobid_2string(job->job_base.jobid, strJobId),
                            job->inst_usage[0].pgid[i]);
            }
        }

        if (ndead == job->inst_usage[0].npgids) {
            log_message(DEBUG, DEBUGSIGNAL, "%s: Job %s pgids all dead", 
                        __func__, jobid_2string(job->job_base.jobid, strJobId));
            return -1;
        }
    } else {
        if (kill(-job->job_base.job_pgid, sig) < 0) {
            log_message(DEBUG, DEBUGSIGNAL, "%s: Job %s kill(%d) group is gone",
                        __func__, jobid_2string(job->job_base.jobid, strJobId),
                        job->job_base.job_pgid);
            return -1;
        }
    }

    log_message(DEBUG, DEBUGSIGNAL, "%s: Job %s child gone, but some pgids still around",
                __func__, jobid_2string(job->job_base.jobid, strJobId));

    return 0;
} // end function kill_jobpg

static int
write_chkpnt_log(char *fn, char *chkpntDir, struct jobEntry *job)
{
    struct record log;
    int cc, eno;
    FILE *fp;

    if (chkpntDir == NULL) {
        return -1;
    }
    if ((fp = fopen(fn, "w")) == NULL) {
        return -1;
    }

    log.type =  RECORD_TYPE_JOB_NEW;
    snprintf(log.version, MAXVERSIONLEN, "%d", QP_XDR_VERSION);

    unsigned int num = 2; /* jobid, userid */

    if (0 < job->job_base.submit->options) {
        num++;
    }

    if (0 < job->job_base.submit->options2) {
        num++;
    }

    num += 2; /* numProcessors, maxNumProcessors */
    num++; /* userName */

    if (0 < job->job_base.signal) {
        num++; /* signal */
    }

    if (0 < job->job_base.job_pgid) {
        num++;
    }

    if (0 < job->job_base.submit->chkpnt_duration) {
        num++;
    }

    num += 3; /* hostFactor, umask, queue */   
    
    if (NULL != job->job_base.submit->resource_requirement
        && '\0' != job->job_base.submit->resource_requirement[0]) {
        num++;
    }

    num += 3; /* fromHost, cwd, subHomeDir */    
    num += 2; /*chkpntDir, chkpntMethod*/
    num += 3;  /* projectName, command, jobFile */

    if (NULL != job->job_base.submit->input_file && '\0' != job->job_base.submit->input_file[0]) {
        num++;
    }

    if (NULL != job->job_base.submit->output_file && '\0' != job->job_base.submit->output_file[0]) {
        num++;
    }    

    if (NULL != job->job_base.submit->error_file && '\0' != job->job_base.submit->error_file[0]) {
        num++;
    }

    if (NULL != job->job_base.submit->mail_user && '\0' != job->job_base.submit->mail_user[0]) {
        num++;
    }

    if (NULL != job->job_base.submit->job_name && '\0' != job->job_base.submit->job_name[0]) {
        num++;
    }

    if (NULL != job->job_base.submit->command && '\0' != job->job_base.submit->command[0]) {
        num++;
    }

    if (job->job_base.num_exec_files > 0) {
        unsigned int i;
        num++; /* num_exec_files */
        for (i = 0; i < job->job_base.num_exec_files; i++) {

            if (NULL != job->job_base.exec_files[i].in_file
                && '\0' != job->job_base.exec_files[i].in_file[0]) {
                num++; /* execution_file.in_file */
            }

            if (NULL != job->job_base.exec_files[i].out_file
                && '\0' != job->job_base.exec_files[i].out_file[0]) {
                num++; /* execution_file.out_file */
            }

            num++; /* execution_file.options */
        }
    }

    if (0 < job->job_base.submit->spec_priority) {
        num++; /* spec_priority */
    }

    if (NULL != job->job_base.submit->login_shell && '\0' != job->job_base.submit->login_shell[0]) {
        num++;
    }

    if (NULL == job->job_base.schedule_host && '\0' != job->job_base.schedule_host[0]) {
        num++;
    }

    log.KVs = (key_value_t *)calloc(num, sizeof(key_value_t));
    if (NULL == log.KVs) {
        log_message(ERR, NOCLASS, STR_FUNC_D_FAIL_M, __func__,
                    "calloc", num*sizeof(key_value_t));
        relaunch();
    }

    log.numKVs = 0;

    log.KVs[log.numKVs].key = RECORD_KEY_JOBID;
    log.KVs[log.numKVs].value = (char *)calloc(32, sizeof(char));
    if (NULL == log.KVs[log.numKVs].value) {
        log_message(ERR, NOCLASS, STR_FUNC_D_FAIL_M, __func__, "calloc", 32);
        relaunch();
    }
    jobid_2longstring( job->job_base.jobid, log.KVs[log.numKVs].value);
    log.numKVs ++;

    log.KVs[log.numKVs].key = RECORD_KEY_USERID;
    log.KVs[log.numKVs].value = (char *)calloc(12, sizeof(char));
    if (NULL == log.KVs[log.numKVs].value) {
        log_message(ERR, NOCLASS, STR_FUNC_D_FAIL_M, __func__, "calloc", 12);
        relaunch();
    }
    snprintf(log.KVs[log.numKVs].value, 12, "%d", job->job_base.submit->uid);
    log.numKVs ++;    

     if (0 < job->job_base.submit->options) {
        log.KVs[log.numKVs].key = RECORD_KEY_OPTIONS;
        log.KVs[log.numKVs].value = (char *)calloc(12, sizeof(char));
        if (NULL == log.KVs[log.numKVs].value) {
            log_message(ERR, NOCLASS, STR_FUNC_D_FAIL_M, __func__, "calloc", 12);
            relaunch();
        }
        snprintf(log.KVs[log.numKVs].value, 12, "%d", job->job_base.submit->options);
        log.numKVs ++;
    }

    if (0 < job->job_base.submit->options2) {
        log.KVs[log.numKVs].key = RECORD_KEY_OPTIONS2;
        log.KVs[log.numKVs].value = (char *)calloc(12, sizeof(char));
        if (NULL == log.KVs[log.numKVs].value) {
            log_message(ERR, NOCLASS, STR_FUNC_D_FAIL_M, __func__, "calloc", 12);
            relaunch();
        }
        snprintf(log.KVs[log.numKVs].value, 12, "%d", job->job_base.submit->options2);
        log.numKVs ++;
    }   

    unsigned int numProc=0;
    unsigned int i;
    for (i=0; i<job->job_base.number_node; i++) {
        numProc += job->job_base.node_execinfo[i].num_slot;
    }    
    log.KVs[log.numKVs].key = RECORD_KEY_PROCESSOR;
    log.KVs[log.numKVs].value = (char *)calloc(12, sizeof(char));
    if (NULL == log.KVs[log.numKVs].value) {
        log_message(ERR, NOCLASS, STR_FUNC_D_FAIL_M, __func__, "calloc", 12);
        relaunch();
    }
    if (job->job_base.submit->options2 & SUBOPT2_DEF_PROCLIMIT) {
        snprintf(log.KVs[log.numKVs].value, 12, "%d", 1);
    } else {
        snprintf(log.KVs[log.numKVs].value, 12, "%d", numProc);
    }
    log.numKVs ++;

    log.KVs[log.numKVs].key = RECORD_KEY_MAX_PROCESSOR;
    log.KVs[log.numKVs].value = (char *)calloc(12, sizeof(char));
    if (NULL == log.KVs[log.numKVs].value) {
        log_message(ERR, NOCLASS, STR_FUNC_D_FAIL_M, __func__, "calloc", 12);
        relaunch();
    }
    if (job->job_base.submit->options2 & SUBOPT2_DEF_PROCLIMIT) {
        snprintf(log.KVs[log.numKVs].value, 12, "%d", 1);
    } else {
        snprintf(log.KVs[log.numKVs].value, 12, "%d",  job->job_base.submit->max_job_slot);
    }
    log.numKVs ++;

    log.KVs[log.numKVs].key = RECORD_KEY_USERNAME;
    log.KVs[log.numKVs].value = copy_string(job->job_base.submit->user);
    log.numKVs ++;

    if (NULL != job->job_base.schedule_host && '\0' != job->job_base.schedule_host[0]) {
        log.KVs[log.numKVs].key = RECORD_KEY_SCHEDULE_HOSTTYPE;
        log.KVs[log.numKVs].value = copy_string(job->job_base.schedule_host);
        log.numKVs ++;
    }   

    if (NULL != job->job_base.submit->login_shell && '\0' != job->job_base.submit->login_shell[0]) {
        log.KVs[log.numKVs].key = RECORD_KEY_LOGIN_SHELL;
        log.KVs[log.numKVs].value = copy_string(job->job_base.submit->login_shell);
        log.numKVs ++;
    }

    if (0 < job->job_base.signal) {
        log.KVs[log.numKVs].key = RECORD_KEY_SIGNAL;
        log.KVs[log.numKVs].value = (char *)calloc(12, sizeof(char));
        if (NULL == log.KVs[log.numKVs].value) {
            log_message(ERR, NOCLASS, STR_FUNC_D_FAIL_M, __func__, "calloc", 12);
            relaunch();
        }
        snprintf(log.KVs[log.numKVs].value, 12, "%d", job->job_base.signal);
        log.numKVs ++;
    }

    log.KVs[log.numKVs].key = RECORD_KEY_UMASK;
    log.KVs[log.numKVs].value = (char *)calloc(12, sizeof(char));
    if (NULL == log.KVs[log.numKVs].value) {
        log_message(ERR, NOCLASS, STR_FUNC_D_FAIL_M, __func__, "calloc", 12);
        relaunch();
    }
    snprintf(log.KVs[log.numKVs].value, 12, "%d", job->job_base.submit->umask);
    log.numKVs ++;

    log.KVs[log.numKVs].key = RECORD_KEY_QUEUE;
    log.KVs[log.numKVs].value = copy_string(job->job_base.submit->queue);
    log.numKVs ++;

    if (NULL != job->job_base.submit->resource_requirement
        && '\0' != job->job_base.submit->resource_requirement) {
        log.KVs[log.numKVs].key = RECORD_KEY_RESREQ;
        log.KVs[log.numKVs].value = copy_string(job->job_base.submit->resource_requirement);
        log.numKVs ++;
    }

    log.KVs[log.numKVs].key = RECORD_KEY_FROM_HOST;
    log.KVs[log.numKVs].value = copy_string(job->job_base.submit->from_node);
    log.numKVs ++;

    log.KVs[log.numKVs].key = RECORD_KEY_CWD;
    log.KVs[log.numKVs].value = copy_string(job->job_base.submit->cwd);
    log.numKVs ++;

    log.KVs[log.numKVs].key = RECORD_KEY_SUBMIT_HOMEDIR;
    log.KVs[log.numKVs].value = copy_string(job->job_base.submit->home_dir);
    log.numKVs ++;

    if (NULL != job->job_base.submit->input_file && '\0' != job->job_base.submit->input_file[0]) {
        log.KVs[log.numKVs].key = RECORD_KEY_SUBMIT_INFILE;
        log.KVs[log.numKVs].value = copy_string(job->job_base.submit->input_file);
        log.numKVs ++;
    }

    if (NULL != job->job_base.submit->output_file && '\0' != job->job_base.submit->output_file[0]) {
        log.KVs[log.numKVs].key = RECORD_KEY_SUBMIT_OUTFILE;
        log.KVs[log.numKVs].value = copy_string(job->job_base.submit->output_file);
        log.numKVs ++;
    }    

    if (NULL != job->job_base.submit->error_file && '\0' != job->job_base.submit->error_file[0]) {
        log.KVs[log.numKVs].key = RECORD_KEY_SUBMIT_ERRFILE;
        log.KVs[log.numKVs].value = copy_string(job->job_base.submit->error_file);
        log.numKVs ++;
    }

    log.KVs[log.numKVs].key = RECORD_KEY_JOB_FILE;
    log.KVs[log.numKVs].value = copy_string(strrchr(job->job_base.submit->script, '/') + 1);
    log.numKVs ++;

    if (NULL != job->job_base.submit->job_name && '\0' != job->job_base.submit->job_name[0]) {
        log.KVs[log.numKVs].key = RECORD_KEY_JOB_NAME;
        log.KVs[log.numKVs].value = copy_string(job->job_base.submit->job_name);
        log.numKVs ++;
    }
    
    log.KVs[log.numKVs].key = RECORD_KEY_COMMAND;
    log.KVs[log.numKVs].value = copy_string(job->job_base.submit->command);
    log.numKVs ++;

    if (NULL != job->job_base.submit->mail_user && '\0' != job->job_base.submit->mail_user[0]) {
        log.KVs[log.numKVs].key = RECORD_KEY_MAIL_USER;
        log.KVs[log.numKVs].value = copy_string(job->job_base.submit->mail_user);
        log.numKVs ++;
    }

    log.KVs[log.numKVs].key = RECORD_KEY_PROJECT_NAME;
    log.KVs[log.numKVs].value = copy_string(job->job_base.submit->project);
    log.numKVs ++;

    if (job->job_base.num_exec_files > 0) {
        log.KVs[log.numKVs].key = RECORD_KEY_NUM_XFILE;
        log.KVs[log.numKVs].value = (char *)calloc(12, sizeof(char));
        if (NULL == log.KVs[log.numKVs].value) {
            log_message(ERR, NOCLASS, STR_FUNC_D_FAIL_M, __func__, "calloc", 12);
            relaunch();
        }
        snprintf(log.KVs[log.numKVs].value, 12, "%d", job->job_base.num_exec_files);
        log.numKVs ++;

        unsigned int i;
        for (i = 0; i < job->job_base.num_exec_files; i++) {

            unsigned int len;
            len = strlen(job->job_base.exec_files[i].in_file)
                  + strlen(job->job_base.exec_files[i].out_file) + 5;

            log.KVs[log.numKVs].key = RECORD_KEY_XFILE;
            log.KVs[log.numKVs].value = (char *)calloc(len+1, sizeof(char));
            if (NULL == log.KVs[log.numKVs].value) {
                log_message(ERR, NOCLASS, STR_FUNC_D_FAIL_M, __func__, "calloc", len+1);
                relaunch();
            }

            if (job->job_base.exec_files[i].direction & XF_OP_SUB2EXEC) {
                snprintf(log.KVs[log.numKVs].value, len, "%s > %s",
                         job->job_base.exec_files[i].in_file, job->job_base.exec_files[i].out_file);
                log.numKVs ++;
            } else if (job->job_base.exec_files[i].direction & XF_OP_SUB2EXEC_APPEND) {
                snprintf(log.KVs[log.numKVs].value, len, "%s >> %s",
                         job->job_base.exec_files[i].in_file, job->job_base.exec_files[i].out_file);
                log.numKVs ++;
            } else if (job->job_base.exec_files[i].direction & XF_OP_EXEC2SUB_APPEND) {
                snprintf(log.KVs[log.numKVs].value, len, "%s << %s",
                         job->job_base.exec_files[i].in_file, job->job_base.exec_files[i].out_file);
                log.numKVs ++;
            } else if (job->job_base.exec_files[i].direction & XF_OP_EXEC2SUB) {
                snprintf(log.KVs[log.numKVs].value, len, "%s < %s",
                         job->job_base.exec_files[i].in_file, job->job_base.exec_files[i].out_file);
                log.numKVs ++;
            }
        }
    }

    if (0 < job->job_base.submit->spec_priority) {
        log.KVs[log.numKVs].key = RECORD_KEY_USER_PRIORITY;
        log.KVs[log.numKVs].value = (char *)calloc(12, sizeof(char));
        if (NULL == log.KVs[log.numKVs].value) {
            log_message(ERR, NOCLASS, STR_FUNC_D_FAIL_M, __func__, "calloc", 12);
            relaunch();
        }
        snprintf(log.KVs[log.numKVs].value, 12, "%d", job->job_base.submit->spec_priority);
        log.numKVs ++;
    }

    log.KVs[log.numKVs].key = RECORD_KEY_CHECKPOINT_DIR;
    log.KVs[log.numKVs].value = copy_string(chkpntDir);
    log.numKVs ++;

    log.KVs[log.numKVs].key = RECORD_KEY_CHECKPOINT_METHOD;
    log.KVs[log.numKVs].value = copy_string(job->job_base.submit->chkpnt_method);
    log.numKVs ++;

    if (0 < job->job_base.submit->chkpnt_duration) {
        log.KVs[log.numKVs].key = RECORD_KEY_CHECKPOINT_PERIOD;
        log.KVs[log.numKVs].value = (char *)calloc(12, sizeof(char));
        if (NULL == log.KVs[log.numKVs].value) {
            log_message(ERR, NOCLASS, STR_FUNC_D_FAIL_M, __func__, "calloc", 12);
            relaunch();
        }
        snprintf(log.KVs[log.numKVs].value, 12, "%d",
                 job->job_base.submit->chkpnt_duration);
        log.numKVs ++;
    }

    if (0 < job->job_base.job_pgid) {
        log.KVs[log.numKVs].key = RECORD_KEY_JOB_PGID;
        log.KVs[log.numKVs].value = (char *)calloc(12, sizeof(char));
        if (NULL == log.KVs[log.numKVs].value) {
            log_message(ERR, NOCLASS, STR_FUNC_D_FAIL_M, __func__, "calloc", 12);
            relaunch();
        }
        snprintf(log.KVs[log.numKVs].value, 12, "%d", job->job_base.job_pgid);
        log.numKVs ++;
    }

    log.log_time = time(NULL);
    cc = write_record(fp, &log);
    eno = errno;

    if (cc == -1) {
        if (fp != NULL) {
            fclose(fp);
            fp = NULL;
        }
        errno = eno;
        return -1;
    }

    log.type = RECORD_TYPE_JOB_START;
    snprintf(log.version, MAXVERSIONLEN, "%d", QP_XDR_VERSION);

    num = 2; /*jobId jobPid*/

    if (job->job_base.number_node > 0) {
        num += 2;   /*number_node, node_exec */
    }

    log.KVs = (key_value_t *)calloc(num, sizeof(key_value_t));
    if (NULL == log.KVs) {
        log_message(ERR, NOCLASS, STR_FUNC_D_FAIL_M, __func__,
                    "calloc", num*sizeof(key_value_t));
        relaunch();
    }

    log.numKVs = 0;

    log.KVs[log.numKVs].key = RECORD_KEY_JOBID;
    log.KVs[log.numKVs].value = (char *)calloc(32, sizeof(char));
    if (NULL == log.KVs[log.numKVs].value) {
        log_message(ERR, NOCLASS, STR_FUNC_D_FAIL_M, __func__, "calloc", 32);
        relaunch();
    }
    jobid_2longstring( job->job_base.jobid, log.KVs[log.numKVs].value);
    log.numKVs ++;
    
    if (0 < job->job_base.number_node) {
        log.KVs[log.numKVs].key = RECORD_KEY_NUM_ASKEDHOST;
        log.KVs[log.numKVs].value = (char *)calloc(12, sizeof(char));
        if (NULL == log.KVs[log.numKVs].value) {
            log_message(ERR, NOCLASS, STR_FUNC_D_FAIL_M, __func__, "calloc", 32);
            relaunch();
        }
        snprintf(log.KVs[log.numKVs].value, 12, "%d", job->job_base.number_node);
        log.numKVs ++;

        unsigned int len = 0;
        len += strlen(job->job_base.node_execinfo[0].host_name) + 1;
        log.KVs[log.numKVs].key = RECORD_KEY_ASKEDHOST;
        log.KVs[log.numKVs].value = (char *)calloc(len, sizeof(char));
        if (NULL == log.KVs[log.numKVs].value) {
            log_message(ERR, NOCLASS, STR_FUNC_D_FAIL_M, __func__, "calloc", len);
            relaunch();
        }
        strcpy(log.KVs[log.numKVs].value, job->job_base.node_execinfo[0].host_name);
        log.numKVs ++;
    }
    
    log.KVs[log.numKVs].key = RECORD_KEY_JOB_PGID;    
    log.KVs[log.numKVs].value = (char *)calloc(12, sizeof(char));    
    if (NULL == log.KVs[log.numKVs].value) {
        log_message(ERR, NOCLASS, STR_FUNC_D_FAIL_M, __func__, "calloc", 12);
        relaunch();
    }
    snprintf(log.KVs[log.numKVs].value, 12, "%d", job->job_base.job_pgid);
    log.numKVs ++;
    
    if (write_record(fp, &log) == -1) {	
        eno = errno;	
        fclose(fp);	
        fp = NULL;
        errno = eno;	
        return -1;   
    }    
    if (fp != NULL) {        
        fclose(fp);    
        fp = NULL;
    }
    return 0;
} // end function write_chkpnt_log

static void
execute_chkpnt(struct jobEntry *job, int chkflags, char *exitfile)
{
    char chkDir[PATH_MAX+1], chkpntDir[PATH_MAX+1], chkpntDirTmp[PATH_MAX+1];
    char oldJobId[32], newJobId[32];

    char errMsg[PATH_MAX+1], msg[PATH_MAX+1];

    char *sp = errMsg;
    int len = sizeof(errMsg);

    FILE *fp;
    char strJobId[32];
    if (NULL == job->job_base.submit->chkpnt_dir || '\0' == job->job_base.submit->chkpnt_dir[0]) {

        if (send_signal_2job(job, SIGKILL, FALSE) == 0) {

            fp = fopen(exitfile, "w");
            if (NULL == fp) {
                snprintf(errMsg, PATH_MAX, "unable to create job <%s> migration status file(%s) due to %m.", 
                         jobid_2string(job->job_base.jobid, strJobId), exitfile);
                goto Error;
            }
            fclose(fp);

            exit(0);
        }
        snprintf(errMsg, PATH_MAX, "unable to send signal to job <%s>.",
                jobid_2string(job->job_base.jobid, strJobId));

        goto Error;
    }

    send_signal_2job(job, SIGCONT, FALSE);

    char *strDir;
    if (((strDir = strrchr(job->job_base.submit->chkpnt_dir, '/')) != NULL) && (islong(strDir+1))) {
        snprintf(chkpntDir, PATH_MAX, "%s", job->job_base.submit->chkpnt_dir);
        *strDir = '\0';
        snprintf(chkDir, PATH_MAX, "%s", job->job_base.submit->chkpnt_dir);
        snprintf(oldJobId, 32, "%s", strDir+1);
        snprintf(newJobId, 32, "%s", jobid_2string(job->job_base.jobid, strJobId));
        *strDir = '/';
    } else {
        snprintf(errMsg, PATH_MAX, "%s: chkpnt directory %s is not full path.",
                 __func__, job->job_base.submit->chkpnt_dir);
        goto Error;
    }

    if (chdir(chkDir) < 0) {
        snprintf(errMsg, PATH_MAX, "%s: %s(%s) failed due to %m.", __func__, "chdir", chkDir);
        goto Error;
    }

    snprintf(chkpntDir, PATH_MAX, "%s/%s", chkDir, newJobId);
    snprintf(chkpntDirTmp, PATH_MAX, "%s.tmp", newJobId);

    char *p;
    if (job->job_base.submit->chkpnt_dir[0] == '/'
        && (p = strrchr(job->job_base.submit->chkpnt_dir, '/'))
            && p != &(job->job_base.submit->chkpnt_dir[0])) {

        *p = '/';
        if (mkdir(chkpntDirTmp, 0700) < 0 && errno != EEXIST) {
            snprintf(errMsg, PATH_MAX, "%s: %s(%s) failed due to %m.",
                     __func__, "mkdir", chkpntDirTmp);
            *p = '/';
            goto Error;
        }
    } else {
        if (mkdir(chkpntDirTmp, 0700) < 0 && errno != EEXIST) {
            snprintf(errMsg, PATH_MAX, "%s: %s(%s) failed due to %m.",
                     __func__, "mkdir", chkpntDirTmp);
            goto Error;
        }
    }

    int chkpntArgc;
    char *chkpntArgv[MAX_ARGS];

    chkpntArgc = 0;
    chkpntArgv[chkpntArgc++] = (char *)"echkpnt";

    chkpntArgv[chkpntArgc++] = (char *)"--dir";
    chkpntArgv[chkpntArgc++] = chkpntDirTmp;

    chkpntArgv[chkpntArgc++] = (char *)"--method";
    chkpntArgv[chkpntArgc++] = job->job_base.submit->chkpnt_method;

    char strJobPGid[20];
    snprintf(strJobPGid, 12, "%d", job->job_base.job_pgid);
    chkpntArgv[chkpntArgc++] = strJobPGid;

    chkpntArgv[chkpntArgc] = NULL;

    char *strExecChkpnt;
    char strChkpntDir[PATH_MAX+1];
    memset(strChkpntDir, 0, PATH_MAX+1);

    strExecChkpnt = get_chkpnt_dir("/echkpnt", strChkpntDir);
    if (NULL == strExecChkpnt) {
        snprintf(errMsg, PATH_MAX, "Unable to find the echkpnt file.");
        remove_dir_withfiles(chkpntDirTmp, TRUE);
        goto Error;
    }

    int fds[2];
    if (0 > pipe(fds)) {
        snprintf(errMsg, PATH_MAX, STR_JOB_FAIL_S_M, __func__,
                 jobid_2string(job->job_base.jobid, strJobId), "pipe");
        remove_dir_withfiles(chkpntDirTmp, TRUE);
        goto Error;
    }

    pid_t pid;
    pid = fork();
    if (0 > pid) {
        log_message(ERR, NOCLASS, STR_JOB_FAIL_S_M, __func__,
                    jobid_2string(job->job_base.jobid, strJobId), "fork");
        remove_dir_withfiles(chkpntDirTmp, TRUE);
        goto Error;
    }

    if (0 == pid) { // child 
        close(fds[0]);
        dup2(fds[1], 2);

        change_uid(guidManager);

        if (!(NULL != baseParams[QP_CHKPNT_ROOT].string_value
              && 0 == strcmp(baseParams[QP_CHKPNT_ROOT].string_value, "Y"))) {

            if (0 > setuid(job->job_base.execute_uid)) {
                fprintf(stderr, STR_JOB_FAIL_S_D_M, __func__,
                        jobid_2string(job->job_base.jobid, strJobId),
                        "setuid", job->job_base.execute_uid);
                fclose(stderr);
                exit(-1);
            }
        }

        execv(strExecChkpnt, chkpntArgv);

        fprintf(stderr, STR_FUNC_S_FAIL_M, __func__, "execv", "echkpnt");
        fclose(stderr);

        exit(-1);
    }

    close(fds[1]);

    pid_t echkpntPid;
    echkpntPid = pid;

    int cc;
    while ((cc = read(fds[0], sp, len)) > 0) {
        sp += cc;
        len -= cc;
    }
    *sp = '\0';

    if ((strlen(errMsg) > 0) && (strstr(errMsg, "Checkpoint done") == NULL)) {
        remove_dir_withfiles(chkpntDirTmp, TRUE);
        goto Error;
    }

    STATUS_WAIT_T status;
    while ((pid = waitpid(echkpntPid, &status, 0)) < 0 && errno == EINTR);

    if (0 > pid) {
        snprintf(errMsg, PATH_MAX, STR_JOB_FAIL_S_D_M, __func__,
                 jobid_2string(job->job_base.jobid, strJobId), "waitpid", (int)pid);
        remove_dir_withfiles(chkpntDirTmp, TRUE);
        goto Error;
    } else {
        if (WIFEXITED(status)) {

            if (WEXITSTATUS(status) != 0) {
                snprintf(msg, PATH_MAX, "job <%s> exited with the exit code %d with <%s>",
                         jobid_2string(job->job_base.jobid, strJobId),
                         WEXITSTATUS(status), errMsg);
                strncpy(errMsg, msg, PATH_MAX);
                remove_dir_withfiles(chkpntDirTmp, TRUE);
                goto Error;
            }
        }
    }

    DIR *dirChkpnt;
    dirChkpnt = opendir(chkpntDirTmp);

    char s[PATH_MAX+1], t[PATH_MAX+1];

    struct dirent *dp;
    while ((dp = readdir(dirChkpnt)) != NULL) {

        if (strcmp(dp->d_name, ".") == 0 || strcmp(dp->d_name, "..") == 0) {
            continue;
        }

        snprintf(s, PATH_MAX, "%s/%s", chkpntDirTmp, dp->d_name);
        snprintf(t, PATH_MAX, "%s/%s", newJobId, dp->d_name);

        if (rename(s, t) == -1) {
            snprintf(errMsg, PATH_MAX, "Unable to rename checkpoint file %s to %s: %s", 
                     s, t, strerror(errno));
            closedir(dirChkpnt);
            goto Error;
        }
    }
    closedir(dirChkpnt);

    if (strcmp(oldJobId, newJobId) != 0) {
        unlink(oldJobId);
    }

    remove_dir_withfiles(chkpntDirTmp, TRUE);

    snprintf(s, PATH_MAX, "%s/chklog", newJobId);
    if (write_chkpnt_log(s, chkDir, job) == -1) {
        snprintf(errMsg, PATH_MAX, "Unable to write to checkpoint log %s/%s", chkDir, s);
        goto Error;
    }

    change_uid(guidManager);

    if ((fp = fopen(exitfile, "w")) == NULL || fclose(fp) == -1) {
        snprintf(errMsg, PATH_MAX, STR_FUNC_S_FAIL_M, __func__, "fopen", exitfile);
        goto Error;
    }
    exit(0);

Error:

    snprintf(msg, PATH_MAX, "We are unable to checkpoint your job %s <%s>.\nThe error is: %s",
             jobid_2string(job->job_base.jobid, strJobId), job->job_base.submit->command, errMsg);

    log_message(ERR, NOCLASS, "%s: %s", __func__, msg);

    if (job->job_base.status & (JOB_STATUS_SSUSP | JOB_STATUS_USUSP)) {
        send_signal_2job(job, SIGSTOP, FALSE);
    }

    exit(-1);
} // end function execute_chkpnt

static char *
job_pgid_2string(struct jobEntry *job)
{
    char *pgids;

    if (0 < job->inst_usage[0].npgids) {

        unsigned int len;
        len = job->inst_usage[0].npgids * 12;

        pgids = (char *)calloc(len+1, sizeof(char));
        if (NULL == pgids) {
            log_message(ERR, NOCLASS, STR_FUNC_D_FAIL_M, __func__, "calloc", len+1);
            relaunch();
        }

        unsigned int i;
        for (i=0; i<job->inst_usage[0].npgids; i++) {
            snprintf(pgids+strlen(pgids), len, "%d ", job->inst_usage[0].pgid[i]);
        }
    } else {

        pgids = (char *)calloc(1, 12);
        if (NULL == pgids) {
            log_message(ERR, NOCLASS, STR_FUNC_D_FAIL_M, __func__, "calloc", 12);
            relaunch();
        }

        snprintf(pgids, 12, "%d", job->job_base.job_pgid);
    }

    return(pgids);
} // end function job_pgid_2string

static char *
job_pid_2string(struct jobEntry *job)
{
    char *pids;

    if (0 < job->inst_usage[0].npids) {

        unsigned int len;
        len = job->inst_usage[0].npids * 12;

        pids = (char *)calloc(len+1, sizeof(char));
        if (NULL == pids) {
            log_message(ERR, NOCLASS, STR_FUNC_D_FAIL_M, __func__, "calloc", len+1);
            relaunch();
        }

        unsigned int i;
        for (i=0; i<job->inst_usage[0].npids; i++) {
            snprintf(pids+strlen(pids), len, "%d ", job->inst_usage[0].pidinfo[i].pid);
        }
    } else {

        pids = (char *)calloc(1, 12);
        if (NULL == pids) {
            log_message(ERR, NOCLASS, STR_FUNC_D_FAIL_M, __func__, "calloc", 12);
            relaunch();
        }

        snprintf(pids, 12, "%d", job->job_base.job_pid);
    }

    return(pids);
} // end function job_pid_2string

/**
 * @brief       Execute signal action script for the job.
 *
 * @param[in]   job          #1: job entry
 * @param[in]   signal       #2: signal value to signal a job
 * @param[in]   command      #3: script to be executed
 * @param[in]   flags        #4: chkpnt flag, kill or stop
 *
 * @retval      0            #1: succeed
 * @retval      -1           #2: failed
 *
 * @note        Fork child to execute signal action script.
 */
static int
execute_signal_command(struct jobEntry *job, int signal, char *command, int flags)
{
    job->job_base.act_signal = signal;

    char strJobId[32];
    log_message(DEBUG, DEBUGSIGNAL, "%s: Send batch system signal %d to job %s",
                __func__, signal, jobid_2string(job->job_base.jobid, strJobId));

    char strJobTmpDir[PATH_MAX+1];
    get_tmpdir_4job(strJobTmpDir, job->job_base.jobid, job->job_base.execute_uid);

    char exitFile[PATH_MAX+1];
    snprintf(exitFile, PATH_MAX, "%s/.%s.%s.%s", strJobTmpDir, job->job_base.submit->script,
             jobid_2longstring(job->job_base.jobid, strJobId), get_exitfile_suffix(signal));

    if (unlink(exitFile) == -1 && errno != ENOENT) {
        log_message(ERR, NOCLASS, STR_JOB_FAIL_S_S_M, __func__,
                    jobid_2string(job->job_base.jobid, strJobId), "unlink", exitFile);
        return -1;
    }

    pid_t pid;
    pid = fork();
    if (0 > pid) {
        log_message(ERR, NOCLASS, STR_JOB_FAIL_S_M, __func__,
                    jobid_2string(job->job_base.jobid, strJobId), "fork");
        return -1;
    } else if (0 == pid) { // child
        
        if ((QUICKPOOL_SIGCHKPNT == signal) || (0 == strcmp(command, "CHKPNT"))) {
            gf_setenv("QP_EXECUTE", (char *)"CHKPNT");
        } else {
            gf_setenv("QP_EXECUTE", (char *)"JOB_CONTROLS");
        }
        
        int cc;
        cc = prepare_4postcmd(job);
        if (cc < 0 && !(cc == -2 && !JOB_STARTED(job)) ) {
            log_message(ERR, NOCLASS, STR_JOB_FAIL_S_M, __func__,
	                jobid_2string(job->job_base.jobid, strJobId), "prepare_4postcmd");

            exit(-1);
        }

        if (QUICKPOOL_SIGCHKPNT == signal) {

            execute_chkpnt(job, flags, exitFile);
        } else if (0 == strcmp(command, "CHKPNT")) {

            if (is_terminate_signal(signal)) {
                execute_chkpnt(job, QP_CHKPNT_KILL, exitFile);
            } else {
                execute_chkpnt(job, QP_CHKPNT_STOP, exitFile);
            }
        } else {
            char strJobId[32];
            snprintf(strJobId, 20, "%d", ARRAY_JOBID(job->job_base.jobid));
            gf_setenv("QP_JOBID", strJobId);
            snprintf(strJobId, 20, "%d", ARRAY_INDEX(job->job_base.jobid));
            gf_setenv("QP_JOBINDEX", strJobId);

            char *pids;
            pids = job_pgid_2string(job);
            gf_setenv("QP_JOBPGIDS", pids);
            FREEUP(pids);

            pids = job_pid_2string(job);
            gf_setenv("QP_JOBPIDS", pids);
            FREEUP(pids);

            char reasons[12];
            snprintf(reasons, 12, "%d", job->susp_reason);
            gf_setenv("QP_SUSP_REASONS", reasons);
            snprintf(reasons, 12, "%d", job->susp_subreason);
            gf_setenv("QP_SUSP_SUBREASONS", reasons);

            pid_t pid1;
            if ((pid1 = fork()) < 0) {
                log_message(ERR, NOCLASS, STR_JOB_FAIL_S_M, __func__,
                            jobid_2string(job->job_base.jobid, strJobId), "fork");
                exit(-1);
            } else if (pid1 == 0) {
                char *myargv[6];

                myargv[0] = (char *)"/bin/sh";
                myargv[1] = (char *)"-c";
                myargv[2] = command;
                myargv[3] = NULL;

                execvp ("/bin/sh", myargv);

                exit(-1);
            }

            pid_t commandPid = pid1;
            STATUS_WAIT_T status;
            while ((pid1 = waitpid(commandPid, &status, 0)) < 0 && errno == EINTR);

            if (pid1 < 0) {
                exit(-1);
            } else {
                if (WEXITSTATUS(status)) {
                    exit(WEXITSTATUS(status));
                }
            }

            FILE *fp;
            if ((fp = fopen(exitFile, "w")) == NULL) {
                exit(-1);
            }

            fclose(fp);
            exit(0);
        }
    }

    // parent
    job->job_base.act_pid = pid;

    return 0;
} // end function execute_signal_command
